#!/usr/bin/env python3
"""
FIO集群测试脚本 - 极简版本
在多个节点上并行运行单个fio测试，并汇总聚合测试结果
"""

import os
import sys
import json
import time
import logging
import argparse
import subprocess
import threading
import signal
from datetime import datetime
from typing import Dict, List, Tuple


class FioClusterTester:
    """FIO集群测试器 - 极简版本"""
    
    def __init__(self, fio_command: str, nodes: List[str], count: int = 1, wait_time: int = 0, log_level: str = "INFO", log_file: str = None):
        """初始化测试器"""
        self.fio_command = fio_command
        self.nodes = nodes
        self.count = count
        self.wait_time = wait_time  # 每轮测试完成后的等待时间（秒）
        self.log_level = log_level
        self.log_file = log_file or f'fio_test_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log'
        self.results = {}
        self.all_test_results = []  # 存储所有测试轮次的结果
        self.interrupted = False  # 标记是否被中断
        self.cleanup_lock = threading.Lock()  # 清理操作的线程锁
        
        self.setup_logging()
    
    def setup_logging(self):
        """设置日志"""
        # 创建根日志记录器
        root_logger = logging.getLogger()
        root_logger.setLevel(getattr(logging, self.log_level.upper()))
        
        # 清除现有的处理器
        for handler in root_logger.handlers[:]:
            root_logger.removeHandler(handler)
        
        # 创建格式化器
        formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
        
        # 文件处理器
        file_handler = logging.FileHandler(self.log_file)
        file_handler.setFormatter(formatter)
        root_logger.addHandler(file_handler)
        
        # 控制台处理器
        console_handler = logging.StreamHandler(sys.stdout)
        console_handler.setFormatter(formatter)
        root_logger.addHandler(console_handler)
        
        # 创建线程锁用于保护日志输出
        self.log_lock = threading.Lock()
        self.logger = logging.getLogger(__name__)
    
    def _log_info(self, message: str, host: str = None):
        """线程安全的日志记录"""
        with self.log_lock:
            if host:
                self.logger.info(f"[{host}] {message}")
            else:
                self.logger.info(message)
    
    def _log_error(self, message: str, host: str = None):
        """线程安全的错误日志记录"""
        with self.log_lock:
            if host:
                self.logger.error(f"[{host}] {message}")
            else:
                self.logger.error(message)
    
    def _log_warning(self, message: str, host: str = None):
        """线程安全的警告日志记录"""
        with self.log_lock:
            if host:
                self.logger.warning(f"[{host}] {message}")
            else:
                self.logger.warning(message)
    
    def _execute_ssh_command(self, host: str, username: str, command: str, port: int = 22) -> Tuple[int, str, str]:
        """执行SSH命令"""
        try:
            # 构建SSH命令（假设已配置SSH免密登录）
            ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=10', 
                      '-p', str(port), f'{username}@{host}', command]
            
            # 执行命令
            result = subprocess.run(ssh_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=10000)
            stdout = result.stdout.decode('utf-8') if result.stdout else ""
            stderr = result.stderr.decode('utf-8') if result.stderr else ""
            return result.returncode, stdout, stderr
            
        except subprocess.TimeoutExpired:
            self.logger.error(f"SSH命令超时: {host}")
            return -1, "", "Command timeout"
        except Exception as e:
            self.logger.error(f"SSH命令执行失败: {e}")
            return -1, "", str(e)
    
    def _run_fio_test_on_node(self, node: str) -> Dict:
        """在单个节点上运行fio测试"""
        host = node
        username = "root"  # 默认用户名
        port = 22  # 默认SSH端口
        
        self._log_info(f"开始在节点上运行fio测试", host)
        
        try:
            # 在日志中打印FIO命令
            self._log_info(f"FIO命令: {self.fio_command}", host)
            
            # 检查fio是否安装
            exit_code, stdout, stderr = self._execute_ssh_command(host, username, 'which fio', port=port)
            if exit_code != 0:
                self._log_warning(f"未安装fio，尝试安装...", host)
                install_cmd = 'sudo apt-get update && sudo apt-get install -y fio'
                exit_code, stdout, stderr = self._execute_ssh_command(host, username, install_cmd, port=port)
                if exit_code != 0:
                    self._log_error(f"安装fio失败: {stderr}", host)
                    return {'host': host, 'error': f'安装fio失败: {stderr}'}
            
            # 运行fio测试
            self._log_info(f"开始执行FIO测试", host)
            
            start_time = time.time()
            exit_code, stdout, stderr = self._execute_ssh_command(host, username, self.fio_command, port=port)
            end_time = time.time()
            
            if exit_code != 0:
                self._log_error(f"FIO测试失败: {stderr}", host)
                return {'host': host, 'error': f'FIO测试失败: {stderr}'}
            
            # 解析fio结果
            try:
                fio_result = json.loads(stdout)
                result = {
                    'host': host,
                    'test_duration': end_time - start_time,
                    'fio_result': fio_result,
                    'timestamp': datetime.now().isoformat()
                }
                
                # 提取关键指标
                jobs = fio_result.get('jobs', [])
                if jobs:
                    job = jobs[0]  # 通常只有一个job
                    read_stats = job.get('read', {})
                    write_stats = job.get('write', {})
                    
                    result['metrics'] = {
                        'read_iops': read_stats.get('iops', 0),
                        'read_bw': read_stats.get('bw', 0),  # KB/s
                        'read_lat_avg': read_stats.get('lat_ns', {}).get('mean', 0) / 1000000,  # ms
                        'write_iops': write_stats.get('iops', 0),
                        'write_bw': write_stats.get('bw', 0),  # KB/s
                        'write_lat_avg': write_stats.get('lat_ns', {}).get('mean', 0) / 1000000,  # ms
                    }
                
                self._log_info(f"FIO测试完成", host)
                return result
                
            except json.JSONDecodeError as e:
                self._log_error(f"FIO结果解析失败: {e}", host)
                return {'host': host, 'error': f'结果解析失败: {e}'}
            
        except Exception as e:
            self._log_error(f"测试过程中发生错误: {e}", host)
            return {'host': host, 'error': str(e)}
    
    def run_cluster_test(self) -> Dict:
        """运行集群测试"""
        if not self.nodes:
            self._log_error("没有指定节点信息")
            return {}
        
        self._log_info(f"开始在 {len(self.nodes)} 个节点上运行fio测试，共 {self.count} 轮")
        
        # 运行多轮测试
        for test_round in range(1, self.count + 1):
            # 检查是否被中断
            if self.interrupted:
                self._log_info("测试被中断，停止执行")
                break
                
            self._log_info(f"开始第 {test_round}/{self.count} 轮测试")
            
            # 使用线程池并行执行测试
            threads = []
            results_lock = threading.Lock()
            round_results = {}
            
            def run_node_test(node):
                # 检查是否被中断
                if self.interrupted:
                    return
                result = self._run_fio_test_on_node(node)
                with results_lock:
                    round_results[node] = result
            
            # 创建并启动线程
            for node in self.nodes:
                if self.interrupted:
                    break
                thread = threading.Thread(target=run_node_test, args=(node,))
                threads.append(thread)
                thread.start()
            
            # 等待所有线程完成
            for thread in threads:
                thread.join()
            
            # 如果被中断，不保存结果
            if self.interrupted:
                self._log_info("测试被中断，不保存当前轮次结果")
                break
            
            # 保存本轮测试结果
            self.all_test_results.append({
                'round': test_round,
                'results': round_results,
                'timestamp': datetime.now().isoformat()
            })
            
            # 打印本轮测试汇总结果
            self._print_round_summary(test_round, round_results)
            
            self._log_info(f"第 {test_round}/{self.count} 轮测试完成")
            
            # 如果不是最后一轮测试，等待指定时间
            if test_round < self.count and self.wait_time > 0:
                print(f"\n等待 {self.wait_time} 秒后开始下一轮测试...")
                self._log_info(f"等待 {self.wait_time} 秒后开始下一轮测试...")
                for remaining in range(self.wait_time, 0, -1):
                    if self.interrupted:
                        print("\n测试被中断，停止等待")
                        self._log_info("测试被中断，停止等待")
                        break
                    print(f"\r等待中... {remaining} 秒", end='', flush=True)
                    time.sleep(1)
                print()  # 换行
                if not self.interrupted:
                    print("等待完成，开始下一轮测试")
                    self._log_info("等待完成，开始下一轮测试")
        
        # 计算所有轮次的平均结果
        self.results = self._calculate_average_results()
        return self.results
    
    def _calculate_average_results(self) -> Dict:
        """计算多轮测试的平均结果"""
        if not self.all_test_results:
            return {}
        
        # 收集所有成功的测试结果
        all_successful_results = []
        for test_round in self.all_test_results:
            round_results = test_round['results']
            successful_results = [r for r in round_results.values() if 'error' not in r and 'metrics' in r]
            all_successful_results.extend(successful_results)
        
        if not all_successful_results:
            return {}
        
        # 按节点分组计算平均值
        node_results = {}
        for node in self.nodes:
            node_metrics = []
            for test_round in self.all_test_results:
                round_results = test_round['results']
                if node in round_results and 'error' not in round_results[node] and 'metrics' in round_results[node]:
                    node_metrics.append(round_results[node]['metrics'])
            
            if node_metrics:
                # 计算该节点所有轮次的平均指标
                avg_metrics = {}
                for metric_name in ['read_iops', 'write_iops', 'read_bw', 'write_bw', 'read_lat_avg', 'write_lat_avg']:
                    values = [m.get(metric_name, 0) for m in node_metrics if m.get(metric_name, 0) > 0]
                    avg_metrics[metric_name] = sum(values) / len(values) if values else 0
                
                # 计算平均测试时间
                test_durations = []
                for test_round in self.all_test_results:
                    round_results = test_round['results']
                    if node in round_results and 'test_duration' in round_results[node]:
                        test_durations.append(round_results[node]['test_duration'])
                avg_duration = sum(test_durations) / len(test_durations) if test_durations else 0
                
                node_results[node] = {
                    'host': node,
                    'metrics': avg_metrics,
                    'test_duration': avg_duration,
                    'test_rounds': len(node_metrics),
                    'timestamp': datetime.now().isoformat()
                }
            else:
                # 如果该节点在所有轮次都失败，记录错误
                node_results[node] = {'host': node, 'error': '所有轮次测试都失败'}
        
        return node_results
    
    def _aggregate_round_results(self, round_results: Dict) -> Dict:
        """聚合单轮测试结果"""
        if not round_results:
            return {}
        
        successful_results = [r for r in round_results.values() if 'error' not in r and 'metrics' in r]
        failed_results = [r for r in round_results.values() if 'error' in r]
        
        if not successful_results:
            return {'error': '本轮测试全部失败'}
        
        # 聚合性能指标
        all_read_iops = [r['metrics']['read_iops'] for r in successful_results if r['metrics']['read_iops'] > 0]
        all_write_iops = [r['metrics']['write_iops'] for r in successful_results if r['metrics']['write_iops'] > 0]
        all_read_bw = [r['metrics']['read_bw'] for r in successful_results if r['metrics']['read_bw'] > 0]
        all_write_bw = [r['metrics']['write_bw'] for r in successful_results if r['metrics']['write_bw'] > 0]
        all_read_lat = [r['metrics']['read_lat_avg'] for r in successful_results if r['metrics']['read_lat_avg'] > 0]
        all_write_lat = [r['metrics']['write_lat_avg'] for r in successful_results if r['metrics']['write_lat_avg'] > 0]
        
        aggregated = {
            'total_nodes': len(round_results),
            'successful_nodes': len(successful_results),
            'failed_nodes': len(failed_results),
            'performance_metrics': {
                # 总计指标
                'total_read_iops': sum(all_read_iops),
                'total_write_iops': sum(all_write_iops),
                'total_read_bw': sum(all_read_bw),
                'total_write_bw': sum(all_write_bw),
                
                # 平均指标
                'avg_read_iops': sum(all_read_iops) / len(all_read_iops) if all_read_iops else 0,
                'avg_write_iops': sum(all_write_iops) / len(all_write_iops) if all_write_iops else 0,
                'avg_read_bw': sum(all_read_bw) / len(all_read_bw) if all_read_bw else 0,
                'avg_write_bw': sum(all_write_bw) / len(all_write_bw) if all_write_bw else 0,
                'avg_read_lat': sum(all_read_lat) / len(all_read_lat) if all_read_lat else 0,
                'avg_write_lat': sum(all_write_lat) / len(all_write_lat) if all_write_lat else 0,
            }
        }
        
        return aggregated
    
    def _print_round_summary(self, round_num: int, round_results: Dict):
        """打印单轮测试汇总结果"""
        aggregated = self._aggregate_round_results(round_results)
        
        if 'error' in aggregated:
            print(f"\n=== 第 {round_num} 轮测试汇总 ===")
            print(f"测试失败: {aggregated['error']}")
            return
        
        metrics = aggregated['performance_metrics']
        print(f"\n=== 第 {round_num} 轮测试汇总 ===")
        print(f"成功节点: {aggregated['successful_nodes']}/{aggregated['total_nodes']}")
        
        if metrics:
            print(f"\n📊 总计指标:")
            print(f"  总读取IOPS: {metrics.get('total_read_iops', 0):.2f}")
            print(f"  总写入IOPS: {metrics.get('total_write_iops', 0):.2f}")
            print(f"  总读取带宽: {metrics.get('total_read_bw', 0):.2f} KB/s ({metrics.get('total_read_bw', 0)/1024:.2f} MB/s)")
            print(f"  总写入带宽: {metrics.get('total_write_bw', 0):.2f} KB/s ({metrics.get('total_write_bw', 0)/1024:.2f} MB/s)")
            
            print(f"\n📈 平均指标:")
            print(f"  平均读取IOPS: {metrics.get('avg_read_iops', 0):.2f}")
            print(f"  平均写入IOPS: {metrics.get('avg_write_iops', 0):.2f}")
            print(f"  平均读取带宽: {metrics.get('avg_read_bw', 0):.2f} KB/s")
            print(f"  平均写入带宽: {metrics.get('avg_write_bw', 0):.2f} KB/s")
            print(f"  平均读取延迟: {metrics.get('avg_read_lat', 0):.2f} ms")
            print(f"  平均写入延迟: {metrics.get('avg_write_lat', 0):.2f} ms")
        
        # 打印各节点详细结果
        print(f"\n=== 各节点详细结果 ===")
        for host, result in round_results.items():
            if 'error' in result:
                print(f"{host}: 失败 - {result['error']}")
            else:
                metrics = result.get('metrics', {})
                print(f"{host}:")
                print(f"  读取IOPS: {metrics.get('read_iops', 0):.2f}")
                print(f"  写入IOPS: {metrics.get('write_iops', 0):.2f}")
                print(f"  读取带宽: {metrics.get('read_bw', 0):.2f} KB/s")
                print(f"  写入带宽: {metrics.get('write_bw', 0):.2f} KB/s")
                print(f"  读取延迟: {metrics.get('read_lat_avg', 0):.2f} ms")
                print(f"  写入延迟: {metrics.get('write_lat_avg', 0):.2f} ms")
                print(f"  测试耗时: {result.get('test_duration', 0):.2f} 秒")
    
    def cleanup_fio_processes(self):
        """清理所有节点上的FIO进程"""
        with self.cleanup_lock:
            if self.interrupted:
                return  # 避免重复清理
            
            self.interrupted = True
            self._log_info("检测到中断信号，开始清理FIO进程...")
            
            # 使用线程池并行清理所有节点
            cleanup_threads = []
            
            def cleanup_node(node):
                try:
                    self._log_info(f"正在清理节点 {node} 上的FIO进程...")
                    
                    # 查找并杀死FIO进程
                    kill_commands = [
                        "pkill -f fio",  # 杀死所有fio相关进程
                        "pkill -9 -f fio",  # 强制杀死fio进程
                        "ps aux | grep fio | grep -v grep | awk '{print $2}' | xargs -r kill -9"  # 备用清理命令
                    ]
                    
                    for cmd in kill_commands:
                        exit_code, stdout, stderr = self._execute_ssh_command(node, "root", cmd, port=22)
                        if exit_code == 0:
                            self._log_info(f"节点 {node} FIO进程清理完成")
                            break
                        else:
                            self._log_warning(f"节点 {node} 清理命令失败: {stderr}")
                    
                except Exception as e:
                    self._log_error(f"清理节点 {node} 时发生错误: {e}")
            
            # 创建并启动清理线程
            for node in self.nodes:
                thread = threading.Thread(target=cleanup_node, args=(node,))
                cleanup_threads.append(thread)
                thread.start()
            
            # 等待所有清理线程完成
            for thread in cleanup_threads:
                thread.join(timeout=10)  # 最多等待10秒
            
            self._log_info("FIO进程清理完成")
    
    def aggregate_results(self) -> Dict:
        """聚合测试结果"""
        if not self.results:
            return {}
        
        successful_results = [r for r in self.results.values() if 'error' not in r]
        failed_results = [r for r in self.results.values() if 'error' in r]
        
        if not successful_results:
            self._log_warning("没有成功的测试结果")
            return {'error': '所有测试都失败了'}
        
        # 计算聚合指标
        aggregated = {
            'test_summary': {
                'total_nodes': len(self.results),
                'successful_nodes': len(successful_results),
                'failed_nodes': len(failed_results),
                'test_count': self.count,
                'test_timestamp': datetime.now().isoformat()
            },
            'performance_metrics': {},
            'node_results': self.results
        }
        
        # 聚合性能指标
        all_read_iops = [r['metrics']['read_iops'] for r in successful_results if 'metrics' in r and r['metrics']['read_iops'] > 0]
        all_write_iops = [r['metrics']['write_iops'] for r in successful_results if 'metrics' in r and r['metrics']['write_iops'] > 0]
        all_read_bw = [r['metrics']['read_bw'] for r in successful_results if 'metrics' in r and r['metrics']['read_bw'] > 0]
        all_write_bw = [r['metrics']['write_bw'] for r in successful_results if 'metrics' in r and r['metrics']['write_bw'] > 0]
        all_read_lat = [r['metrics']['read_lat_avg'] for r in successful_results if 'metrics' in r and r['metrics']['read_lat_avg'] > 0]
        all_write_lat = [r['metrics']['write_lat_avg'] for r in successful_results if 'metrics' in r and r['metrics']['write_lat_avg'] > 0]
        
        # 计算总体汇总指标
        if all_read_iops or all_write_iops or all_read_bw or all_write_bw:
            aggregated['performance_metrics'] = {
                # 总计指标
                'total_read_iops': sum(all_read_iops),
                'total_write_iops': sum(all_write_iops),
                'total_read_bw': sum(all_read_bw),
                'total_write_bw': sum(all_write_bw),
                
                # 平均指标
                'avg_read_iops': sum(all_read_iops) / len(all_read_iops) if all_read_iops else 0,
                'avg_write_iops': sum(all_write_iops) / len(all_write_iops) if all_write_iops else 0,
                'avg_read_bw': sum(all_read_bw) / len(all_read_bw) if all_read_bw else 0,
                'avg_write_bw': sum(all_write_bw) / len(all_write_bw) if all_write_bw else 0,
                'avg_read_lat': sum(all_read_lat) / len(all_read_lat) if all_read_lat else 0,
                'avg_write_lat': sum(all_write_lat) / len(all_write_lat) if all_write_lat else 0,
                
                # 最小值
                'min_read_iops': min(all_read_iops) if all_read_iops else 0,
                'min_write_iops': min(all_write_iops) if all_write_iops else 0,
                'min_read_bw': min(all_read_bw) if all_read_bw else 0,
                'min_write_bw': min(all_write_bw) if all_write_bw else 0,
                'min_read_lat': min(all_read_lat) if all_read_lat else 0,
                'min_write_lat': min(all_write_lat) if all_write_lat else 0,
                
                # 最大值
                'max_read_iops': max(all_read_iops) if all_read_iops else 0,
                'max_write_iops': max(all_write_iops) if all_write_iops else 0,
                'max_read_bw': max(all_read_bw) if all_read_bw else 0,
                'max_write_bw': max(all_write_bw) if all_write_bw else 0,
                'max_read_lat': max(all_read_lat) if all_read_lat else 0,
                'max_write_lat': max(all_write_lat) if all_write_lat else 0,
            }
        
        return aggregated
    
    def save_results(self, output_file: str = None):
        """保存测试结果"""
        if not output_file:
            output_file = f'fio_cluster_results_{datetime.now().strftime("%Y%m%d_%H%M%S")}.txt'
        
        aggregated_results = self.aggregate_results()
        
        try:
            with open(output_file, 'w', encoding='utf-8') as f:
                # 写入测试摘要
                if 'error' in aggregated_results:
                    f.write(f"测试失败: {aggregated_results['error']}\n")
                    return
                
                summary = aggregated_results['test_summary']
                f.write("=== FIO集群测试摘要 ===\n")
                f.write(f"测试时间: {summary['test_timestamp']}\n")
                f.write(f"FIO命令: {self.fio_command}\n")
                f.write(f"测试轮次: {summary['test_count']}\n")
                f.write(f"总节点数: {summary['total_nodes']}\n")
                f.write(f"成功节点: {summary['successful_nodes']}\n")
                f.write(f"失败节点: {summary['failed_nodes']}\n\n")
                
                # 写入性能指标
                if aggregated_results['performance_metrics']:
                    metrics = aggregated_results['performance_metrics']
                    f.write("=== 集群总体性能指标汇总 ===\n\n")
                    
                    # 总计指标
                    f.write("📊 总计指标:\n")
                    f.write(f"  总读取IOPS: {metrics.get('total_read_iops', 0):.2f}\n")
                    f.write(f"  总写入IOPS: {metrics.get('total_write_iops', 0):.2f}\n")
                    f.write(f"  总读取带宽: {metrics.get('total_read_bw', 0):.2f} KB/s ({metrics.get('total_read_bw', 0)/1024:.2f} MB/s)\n")
                    f.write(f"  总写入带宽: {metrics.get('total_write_bw', 0):.2f} KB/s ({metrics.get('total_write_bw', 0)/1024:.2f} MB/s)\n\n")
                    
                    # 平均指标
                    f.write("📈 平均指标:\n")
                    f.write(f"  平均读取IOPS: {metrics.get('avg_read_iops', 0):.2f}\n")
                    f.write(f"  平均写入IOPS: {metrics.get('avg_write_iops', 0):.2f}\n")
                    f.write(f"  平均读取带宽: {metrics.get('avg_read_bw', 0):.2f} KB/s\n")
                    f.write(f"  平均写入带宽: {metrics.get('avg_write_bw', 0):.2f} KB/s\n")
                    f.write(f"  平均读取延迟: {metrics.get('avg_read_lat', 0):.2f} ms\n")
                    f.write(f"  平均写入延迟: {metrics.get('avg_write_lat', 0):.2f} ms\n\n")
                    
                
                # 写入各节点详细结果
                f.write("=== 各节点详细结果 ===\n")
                for host, result in self.results.items():
                    if 'error' in result:
                        f.write(f"{host}: 失败 - {result['error']}\n")
                    else:
                        metrics = result.get('metrics', {})
                        f.write(f"{host}:\n")
                        f.write(f"  读取IOPS: {metrics.get('read_iops', 0):.2f}\n")
                        f.write(f"  写入IOPS: {metrics.get('write_iops', 0):.2f}\n")
                        f.write(f"  读取带宽: {metrics.get('read_bw', 0):.2f} KB/s\n")
                        f.write(f"  写入带宽: {metrics.get('write_bw', 0):.2f} KB/s\n")
                        f.write(f"  读取延迟: {metrics.get('read_lat_avg', 0):.2f} ms\n")
                        f.write(f"  写入延迟: {metrics.get('write_lat_avg', 0):.2f} ms\n")
                        f.write(f"  测试耗时: {result.get('test_duration', 0):.2f} 秒\n\n")
            
            self._log_info(f"测试结果已保存到: {output_file}")
        except Exception as e:
            self._log_error(f"保存结果失败: {e}")
    
    def print_summary(self):
        """打印测试摘要"""
        aggregated = self.aggregate_results()
        
        if 'error' in aggregated:
            print(f"测试失败: {aggregated['error']}")
            return
        
        summary = aggregated['test_summary']
        print(f"\n{'='*60}")
        print(f"=== FIO集群测试最终汇总 (所有轮次平均值) ===")
        print(f"{'='*60}")
        print(f"测试时间: {summary['test_timestamp']}")
        print(f"FIO命令: {self.fio_command}")
        print(f"测试轮次: {summary['test_count']}")
        print(f"总节点数: {summary['total_nodes']}")
        print(f"成功节点: {summary['successful_nodes']}")
        print(f"失败节点: {summary['failed_nodes']}")
        
        if aggregated['performance_metrics']:
            metrics = aggregated['performance_metrics']
            print(f"\n=== 集群总体性能指标汇总 (平均值) ===")
            
            # 总计指标
            print(f"\n📊 总计指标 (平均值):")
            print(f"  总读取IOPS: {metrics.get('total_read_iops', 0):.2f}")
            print(f"  总写入IOPS: {metrics.get('total_write_iops', 0):.2f}")
            print(f"  总读取带宽: {metrics.get('total_read_bw', 0):.2f} KB/s ({metrics.get('total_read_bw', 0)/1024:.2f} MB/s)")
            print(f"  总写入带宽: {metrics.get('total_write_bw', 0):.2f} KB/s ({metrics.get('total_write_bw', 0)/1024:.2f} MB/s)")
            
            # 平均指标
            print(f"\n📈 平均指标 (平均值):")
            print(f"  平均读取IOPS: {metrics.get('avg_read_iops', 0):.2f}")
            print(f"  平均写入IOPS: {metrics.get('avg_write_iops', 0):.2f}")
            print(f"  平均读取带宽: {metrics.get('avg_read_bw', 0):.2f} KB/s")
            print(f"  平均写入带宽: {metrics.get('avg_write_bw', 0):.2f} KB/s")
            print(f"  平均读取延迟: {metrics.get('avg_read_lat', 0):.2f} ms")
            print(f"  平均写入延迟: {metrics.get('avg_write_lat', 0):.2f} ms")
            
            # # 最小值指标
            # print(f"\n📉 最小值指标:")
            # print(f"  最小读取IOPS: {metrics.get('min_read_iops', 0):.2f}")
            # print(f"  最小写入IOPS: {metrics.get('min_write_iops', 0):.2f}")
            # print(f"  最小读取带宽: {metrics.get('min_read_bw', 0):.2f} KB/s")
            # print(f"  最小写入带宽: {metrics.get('min_write_bw', 0):.2f} KB/s")
            # print(f"  最小读取延迟: {metrics.get('min_read_lat', 0):.2f} ms")
            # print(f"  最小写入延迟: {metrics.get('min_write_lat', 0):.2f} ms")
            
            # # 最大值指标
            # print(f"\n📈 最大值指标:")
            # print(f"  最大读取IOPS: {metrics.get('max_read_iops', 0):.2f}")
            # print(f"  最大写入IOPS: {metrics.get('max_write_iops', 0):.2f}")
            # print(f"  最大读取带宽: {metrics.get('max_read_bw', 0):.2f} KB/s")
            # print(f"  最大写入带宽: {metrics.get('max_write_bw', 0):.2f} KB/s")
            # print(f"  最大读取延迟: {metrics.get('max_read_lat', 0):.2f} ms")
            # print(f"  最大写入延迟: {metrics.get('max_write_lat', 0):.2f} ms")
        
        # 注意：各节点详细结果已在每轮测试汇总时显示，最终汇总只显示总体统计


def main():
    """主函数"""
    parser = argparse.ArgumentParser(description='FIO集群测试工具 - 极简版本')
    parser.add_argument('--fio-command', required=True, help='FIO命令')
    parser.add_argument('--nodes', nargs='+', required=True, help='节点列表（用空格分隔）')
    parser.add_argument('--count', type=int, default=1, help='测试次数，默认为1次')
    parser.add_argument('--wait-time', type=int, default=120, help='每轮测试完成后的等待时间（秒），默认为120秒')
    parser.add_argument('--output', '-o', help='输出结果文件路径')
    parser.add_argument('--log-level', default='INFO', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'], help='日志级别')
    parser.add_argument('--log-file', help='日志文件路径')
    
    args = parser.parse_args()
    
    # 创建测试器
    tester = FioClusterTester(
        fio_command=args.fio_command,
        nodes=args.nodes,
        count=args.count,
        wait_time=args.wait_time,
        log_level=args.log_level,
        log_file=args.log_file
    )
    
    # 设置信号处理器
    def signal_handler(signum, frame):
        print(f"\n收到信号 {signum}，正在清理FIO进程...")
        tester.cleanup_fio_processes()
        print("清理完成，程序退出")
        sys.exit(1)
    
    # 注册信号处理器
    signal.signal(signal.SIGINT, signal_handler)   # Ctrl+C
    signal.signal(signal.SIGTERM, signal_handler)  # 终止信号
    
    # 运行测试
    try:
        # 运行测试
        results = tester.run_cluster_test()
        
        # 如果测试被中断，不保存结果
        if tester.interrupted:
            print("测试被中断，不保存结果")
            return
        
        # 保存结果
        tester.save_results(args.output)
        
        # 打印摘要
        tester.print_summary()
        
    except KeyboardInterrupt:
        print("\n测试被用户中断")
        tester.cleanup_fio_processes()
        sys.exit(1)
    except Exception as e:
        print(f"测试过程中发生错误: {e}")
        tester.cleanup_fio_processes()
        sys.exit(1)


if __name__ == '__main__':
    main()

